<!DOCTYPE html>
<html lang="zh-cn">
<head>
  <meta charset="utf-8">
  <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1">
  <title>一次不太成功的搬砖（上）：爬取法定传染病疫情月报数据 - Libido Chateau</title>
  <meta name="renderer" content="webkit" />
<meta name="viewport" content="width=device-width, initial-scale=1, maximum-scale=1"/>

<meta http-equiv="Cache-Control" content="no-transform" />
<meta http-equiv="Cache-Control" content="no-siteapp" />

<meta name="theme-color" content="#f8f5ec" />
<meta name="msapplication-navbutton-color" content="#f8f5ec">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="#f8f5ec">


<meta name="author" content="迷幻主义搬砖号子" /><meta name="description" content="卫计委每月发布全国法定传染病疫情公告，包含甲乙丙三类传染病的发病和死亡统计。本打算趁清明假期（什么鬼），用rvest包写个简单的脚本，爬下来分析玩。不料遇到许多莫名其妙的坑。最后，好好一个【爬虫工程】变成了【批处理下载】，十分胸闷。拜托卫计委要继续提高自己的姿势水平。" /><meta name="keywords" content="法定传染病, 疫情报告, 爬虫" />


<meta name="baidu-site-verification" content="k5CktWnwfR" />
<meta name="google-site-verification" content="lO7fVN8Jm2sjiBOcR2knmNBKjtWcxS6KbuZYl2yELwA" />


<meta name="generator" content="Hugo 0.108.0 with theme even" />


<link rel="canonical" href="https://madlogos.github.io/post/infectdis_stat1/" />
<link rel="apple-touch-icon" sizes="180x180" href="/apple-touch-icon.png">
<link rel="icon" type="image/png" sizes="32x32" href="/favicon-32x32.png">
<link rel="icon" type="image/png" sizes="16x16" href="/favicon-16x16.png">
<link rel="manifest" href="/manifest.json">
<link rel="mask-icon" href="/safari-pinned-tab.svg" color="#5bbad5">

<script async src="//busuanzi.ibruce.info/busuanzi/2.3/busuanzi.pure.mini.js"></script>

<link href="/sass/main.min.67ad89b7ca43747ca0757fa0c720e4a26dfb092d0c625a7e1f3b937263f7d48d.css" rel="stylesheet">
<link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@3.5.7/dist/jquery.fancybox.min.css" integrity="sha256-Vzbj7sDDS/woiFS3uNKo8eIuni59rjyNGtXfstRzStA=" crossorigin="anonymous">
<link rel="stylesheet" href="/css/custom.css">


<meta property="og:title" content="一次不太成功的搬砖（上）：爬取法定传染病疫情月报数据" />
<meta property="og:description" content="卫计委每月发布全国法定传染病疫情公告，包含甲乙丙三类传染病的发病和死亡统计。本打算趁清明假期（什么鬼），用rvest包写个简单的脚本，爬下来分析玩。不料遇到许多莫名其妙的坑。最后，好好一个【爬虫工程】变成了【批处理下载】，十分胸闷。拜托卫计委要继续提高自己的姿势水平。" />
<meta property="og:type" content="article" />
<meta property="og:url" content="https://madlogos.github.io/post/infectdis_stat1/" /><meta property="article:section" content="post" />
<meta property="article:published_time" content="2017-04-07T00:00:00+00:00" />
<meta property="article:modified_time" content="2021-05-09T21:28:26+08:00" />


<meta itemprop="name" content="一次不太成功的搬砖（上）：爬取法定传染病疫情月报数据">
<meta itemprop="description" content="卫计委每月发布全国法定传染病疫情公告，包含甲乙丙三类传染病的发病和死亡统计。本打算趁清明假期（什么鬼），用rvest包写个简单的脚本，爬下来分析玩。不料遇到许多莫名其妙的坑。最后，好好一个【爬虫工程】变成了【批处理下载】，十分胸闷。拜托卫计委要继续提高自己的姿势水平。"><meta itemprop="datePublished" content="2017-04-07T00:00:00+00:00" />
<meta itemprop="dateModified" content="2021-05-09T21:28:26+08:00" />
<meta itemprop="wordCount" content="4407">
<meta itemprop="keywords" content="传染病,疫情报告,rvest," /><meta name="twitter:card" content="summary"/>
<meta name="twitter:title" content="一次不太成功的搬砖（上）：爬取法定传染病疫情月报数据"/>
<meta name="twitter:description" content="卫计委每月发布全国法定传染病疫情公告，包含甲乙丙三类传染病的发病和死亡统计。本打算趁清明假期（什么鬼），用rvest包写个简单的脚本，爬下来分析玩。不料遇到许多莫名其妙的坑。最后，好好一个【爬虫工程】变成了【批处理下载】，十分胸闷。拜托卫计委要继续提高自己的姿势水平。"/>

<!--[if lte IE 9]>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/classlist/1.1.20170427/classList.min.js"></script>
<![endif]-->

<!--[if lt IE 9]>
  <script src="https://cdn.jsdelivr.net/npm/html5shiv@3.7.3/dist/html5shiv.min.js"></script>
  <script src="https://cdn.jsdelivr.net/npm/respond.js@1.4.2/dest/respond.min.js"></script>
<![endif]-->

</head>
<body>
  <div id="mobile-navbar" class="mobile-navbar">
  <div class="mobile-header-logo">
    <a href="/" class="logo">Libido Chateau</a>
  </div>
  <div class="mobile-navbar-icon">
    <span></span>
    <span></span>
    <span></span>
  </div>
</div>
<nav id="mobile-menu" class="mobile-menu slideout-menu">
  <ul class="mobile-menu-list">
    <a href="/">
        <li class="mobile-menu-item">首页</li>
      </a><a href="/post/">
        <li class="mobile-menu-item">归档</li>
      </a><a href="/tags/">
        <li class="mobile-menu-item">标签</li>
      </a><a href="/categories/">
        <li class="mobile-menu-item">分类</li>
      </a><a href="/about/">
        <li class="mobile-menu-item">关于</li>
      </a>
  </ul>

  


</nav>

  <div class="container" id="mobile-panel">
    <header id="header" class="header">
        <div class="logo-wrapper">
  <a href="/" class="logo">Libido Chateau</a>
</div>





<nav class="site-navbar">
  <ul id="menu" class="menu">
    <li class="menu-item">
        <a class="menu-item-link" href="/">首页</a>
      </li><li class="menu-item">
        <a class="menu-item-link" href="/post/">归档</a>
      </li><li class="menu-item">
        <a class="menu-item-link" href="/tags/">标签</a>
      </li><li class="menu-item">
        <a class="menu-item-link" href="/categories/">分类</a>
      </li><li class="menu-item">
        <a class="menu-item-link" href="/about/">关于</a>
      </li>
  </ul>
</nav>

    </header>

    <main id="main" class="main">
      <div class="content-wrapper">
        <div id="content" class="content">
          <article class="post">
    
    <header class="post-header">
      <h1 class="post-title">一次不太成功的搬砖（上）：爬取法定传染病疫情月报数据</h1>

      <div class="post-meta">
        <span class="post-time"> 2017-04-07 </span>
        <div class="post-category">
            <a href="/categories/%E6%95%B0%E6%8D%AE/"> 数据 </a>
            </div>
          <span class="more-meta"> 约 4407 字 </span>
          <span class="more-meta"> 预计阅读 9 分钟 </span>
        <span id="busuanzi_container_page_pv" class="more-meta"> <span id="busuanzi_value_page_pv"><img src="/img/spinner.svg" alt="spinner.svg"/></span> 次阅读 </span>
      </div>
    </header>

    <div class="post-toc" id="post-toc">
  <h2 class="post-toc-title">文章目录</h2>
  <div class="post-toc-content always-active">
    <nav id="TableOfContents">
  <ul>
    <li><a href="#带怀旧色彩的源起">带怀旧色彩的源起</a></li>
    <li><a href="#搬砖设想">搬砖设想</a></li>
    <li><a href="#爬目录">爬目录</a>
      <ul>
        <li><a href="#rvest"><code>rvest</code></a></li>
      </ul>
    </li>
    <li><a href="#爬网页">爬网页</a></li>
    <li><a href="#解析发病和死亡总数">解析发病和死亡总数</a></li>
    <li><a href="#抽取各明细病种数据">抽取各明细病种数据</a></li>
  </ul>
</nav>
  </div>
</div>
    <div class="post-content">
      <h2 id="带怀旧色彩的源起">带怀旧色彩的源起</h2>
<p>清明节跑去一个休闲浴场<del>鬼混</del>，在电影厅懒散地看掉了《生化危机6》。场地很豪华（但我就是不透露门牌地址），然而剧情不怎么样——女主光环实在太亮了。倒是病毒-丧尸-疫苗的急性传染病建模设定引起了我的一些职业回忆。</p>
<p>毕业后，我曾在基层疾控中心干过一年多，主要做疫苗接种规划和传染病控制。除了定期不定期地出外勤下现场，就是统计数字、写报告、汇编材料。这些数字沿着行政金字塔的梯级层层上卷，最终汇入国家卫生部疾控局官方报表的大海中。</p>
<p>说是大海，视觉上其实就是类似这样的一张表格：</p>
<!-- {% raw %} -->
<figure class="center"><img src="https://gh-1251443721.cos.ap-chengdu.myqcloud.com/2017/0406/example_infectdis_report.png" width="60%"/><figcaption>
            <h4>图 | 法定传染病统计表</h4>
        </figcaption>
</figure>
<!-- {% endraw %} -->
<p>一晃很多年过去了。籍着这个由头，我又登上了卫生部（现在叫卫计委了，早晚改回卫生部）的官网，那感觉就像——拜会一个久寓故居，新近敲了墙、刷了房门的老派的朋友。那些月报还原封不动，化石一样静静地躺在信息动态里。</p>
<!-- {% raw %} -->
<figure class="center"><img src="https://gh-1251443721.cos.ap-chengdu.myqcloud.com/2017/0406/nhfpc_infectdis_news.png"/><figcaption>
            <h4>图 | 卫计委传染病控制动态</h4>
        </figcaption>
</figure>
<!-- {% endraw %} -->
<!-- {% raw %} -->
<figure class="center"><img src="https://gh-1251443721.cos.ap-chengdu.myqcloud.com/2017/0406/example_infectdis_reporttxt.png"/><figcaption>
            <h4>图 | 法定传染病月报</h4>
        </figcaption>
</figure>
<!-- {% endraw %} -->
<p>这种格式报告，行文和结构都很固定，特别适合用机器人来自动生成。比如最新这期，正文就包括了发病、死亡合计总数，以及甲乙丙类各自的发病、死亡数。明细数据放在附表里。掐指一数，从2004年到现在，卫计委也积攒了140多份月报，不少了。何不爬下来看看？</p>
<p>所以，尽管当时身还在浴场，但心在砖场了，已经！</p>
<h2 id="搬砖设想">搬砖设想</h2>
<p>搬砖虽然是个贱活儿，但也要讲技巧。好在从技术角度，爬这些页面是再简单不过的事。只需要两步就完了：</p>
<ol>
<li>把所有月报页的链接抓到</li>
<li>顺着这些链接把所有页面源码都爬下来</li>
</ol>
<p>更好的消息是所有页面都是静态的。所以只要用<code>rvest</code>就够了，整页爬下来，所有信息就包含在html里面（事实上并不是）。</p>
<p>爬到所有月报页后，解析内容又是两步走：</p>
<ol>
<li>把正文里的发病/死亡总数抽出来，跑个时序图看看周期性</li>
<li>把附表里的内容抽出来，分病种跑些分析</li>
</ol>
<p>Easy as a pie！等我一盏茶的功夫，我去去就来。（结果茶馊了都没能回来）</p>
<h2 id="爬目录">爬目录</h2>
<p>“传染病预防控制”这个分类的URL是很有规律的。第一页是<a href="http://www.nhfpc.gov.cn/jkj/s2907/new_list.shtml">http://www.nhfpc.gov.cn/jkj/s2907/new_list.shtml</a>，第二页就是<a href="http://www.nhfpc.gov.cn/jkj/s2907/new_list_2.shtml">http://www.nhfpc.gov.cn/jkj/s2907/new_list_2.shtml</a>。也就是说23个目录页拼一下就出来了：</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-r" data-lang="r"><span class="line"><span class="cl"><span class="n">urls</span> <span class="o">&lt;-</span> <span class="nf">paste0</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">    <span class="s">&#34;http://www.nhfpc.gov.cn/jkj/s2907/new_list&#34;</span><span class="p">,</span>
</span></span><span class="line"><span class="cl">    <span class="nf">c</span><span class="p">(</span><span class="s">&#34;&#34;</span><span class="p">,</span> <span class="nf">paste0</span><span class="p">(</span><span class="s">&#34;_&#34;</span><span class="p">,</span> <span class="m">2</span><span class="o">:</span><span class="m">23</span><span class="p">)),</span> <span class="s">&#34;.shtml&#34;</span><span class="p">)</span>
</span></span></code></pre></td></tr></table>
</div>
</div><h3 id="rvest"><code>rvest</code></h3>
<p>有了URL，就可以爬源码了。当然可以把网页当文本，直接<code>readLines</code>，然后拿<code>XML</code>包写解析规则。但我们学R图什么？还不就是<strong>免费+有很多包方便偷懒</strong>？对于静态网页，当然毫不犹豫<code>rvest</code>。</p>
<p><code>rvest</code>的核心函数是<code>read_html</code>、<code>html_nodes</code>、<code>html_text</code>和<code>html_table</code>。</p>
<ul>
<li><code>read_html</code>很好理解，把页面读进来。这个页面会被封装为一个<code>xml_nodes</code>对象。</li>
<li><code>html_nodes</code>则负责从<code>xml_nodes</code>对象中提取某个节点的内容，封装成<code>xml_nodeset</code>对象。</li>
<li>进一步，如果要把里面的内容都当做文本提出来，用<code>html_text</code>。</li>
<li>有表格（&lt;table&gt;&hellip;&lt;/table&gt;）的话，用<code>html_table</code>，直接输出梦寐以求的data.frame。</li>
</ul>
<p>唯一费解的概念也就是“节点”。但只要对html和xml略有了解，就很容易理解。一个html文件的典型结构是</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span><span class="lnt">5
</span><span class="lnt">6
</span><span class="lnt">7
</span><span class="lnt">8
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-html" data-lang="html"><span class="line"><span class="cl"><span class="p">&lt;</span><span class="nt">html</span><span class="p">&gt;</span>
</span></span><span class="line"><span class="cl">  <span class="p">&lt;</span><span class="nt">head</span><span class="p">&gt;</span>
</span></span><span class="line"><span class="cl">    <span class="p">&lt;</span><span class="nt">title</span><span class="p">&gt;</span>...<span class="p">&lt;/</span><span class="nt">title</span><span class="p">&gt;</span>
</span></span><span class="line"><span class="cl">  <span class="p">&lt;/</span><span class="nt">head</span><span class="p">&gt;</span>
</span></span><span class="line"><span class="cl">  <span class="p">&lt;</span><span class="nt">body</span><span class="p">&gt;</span>
</span></span><span class="line"><span class="cl">    <span class="p">&lt;</span><span class="nt">h1</span><span class="p">&gt;</span>...<span class="p">&lt;/</span><span class="nt">h1</span><span class="p">&gt;</span>
</span></span><span class="line"><span class="cl">  <span class="p">&lt;/</span><span class="nt">body</span><span class="p">&gt;</span>
</span></span><span class="line"><span class="cl"><span class="p">&lt;/</span><span class="nt">html</span><span class="p">&gt;</span>
</span></span></code></pre></td></tr></table>
</div>
</div><p>从缩进结构可以看出，<code>html</code>是根节点，下一级是两个元素子节点<code>head</code>和<code>body</code>，<code>head</code>的子节点是<code>title</code>，<code>body</code>的子节点是<code>h1</code>，它们还可以有文本、属性或注释子节点。以此类推。</p>
<p>怎么看页面节点结构呢？用Chrome访问目录页，F12查看文档结构，或者右键“查看页面源代码”。</p>
<p>每一篇信息动态都在一个列表节点&lt;li&gt;里，最终都被包进一个无序列表父节点&lt;ul&gt;里。这个无序列表元素的类型是“zxxx_list”（果然还是万能的拼音首字母命名法，“资讯信息”）。所以拿到这个节点后，提取目录信息就很简单了：&lt;a&gt;节点内有标题和链接，&lt;span ml&gt;节点内有发布日期。</p>
<!-- {% raw %} -->
<figure class="center"><img src="https://gh-1251443721.cos.ap-chengdu.myqcloud.com/2017/0406/sourcecode_dir.png"/><figcaption>
            <h4>图 | 网页源码</h4>
        </figcaption>
</figure>
<!-- {% endraw %} -->
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-r" data-lang="r"><span class="line"><span class="cl"><span class="nf">library</span><span class="p">(</span><span class="n">rvest</span><span class="p">)</span>
</span></span><span class="line"><span class="cl"><span class="c1"># 构造一个提取单个目录页信息的函数</span>
</span></span><span class="line"><span class="cl"><span class="n">getTOC</span> <span class="o">&lt;-</span> <span class="nf">function</span><span class="p">(</span><span class="n">url</span><span class="p">){</span>
</span></span><span class="line"><span class="cl">  <span class="c1">## Args</span>
</span></span><span class="line"><span class="cl">  <span class="c1">##   url: 单个网页/网址</span>
</span></span><span class="line"><span class="cl">
</span></span><span class="line"><span class="cl">    <span class="c1"># 读取网页，构成xml_nodes</span>
</span></span><span class="line"><span class="cl">    <span class="n">html</span> <span class="o">&lt;-</span> <span class="nf">read_html</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
</span></span><span class="line"><span class="cl">    <span class="c1"># 从html对象提取ul.zxxx_list节点，只要第一个元素</span>
</span></span><span class="line"><span class="cl">    <span class="n">cast</span> <span class="o">&lt;-</span> <span class="nf">html_nodes</span><span class="p">(</span><span class="n">html</span><span class="p">,</span> <span class="s">&#34;ul.zxxx_list&#34;</span><span class="p">)</span><span class="n">[[1]]</span>
</span></span><span class="line"><span class="cl">    <span class="c1"># 从cast中提取所有&lt;a&gt;节点信息</span>
</span></span><span class="line"><span class="cl">    <span class="n">lists</span> <span class="o">&lt;-</span> <span class="nf">html_nodes</span><span class="p">(</span><span class="n">cast</span><span class="p">,</span> <span class="s">&#34;a&#34;</span><span class="p">)</span>
</span></span><span class="line"><span class="cl">    <span class="c1"># 从cast中提取所有&lt;ml&gt;节点信息</span>
</span></span><span class="line"><span class="cl">    <span class="n">dop</span> <span class="o">&lt;-</span> <span class="nf">html_nodes</span><span class="p">(</span><span class="n">cast</span><span class="p">,</span> <span class="s">&#34;span.ml&#34;</span><span class="p">)</span>
</span></span><span class="line"><span class="cl">    <span class="c1"># 最后得到一个3列数据框，包括链接、标题和发布日期</span>
</span></span><span class="line"><span class="cl">    <span class="nf">return</span><span class="p">(</span><span class="nf">data.frame</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">        <span class="n">href</span><span class="o">=</span><span class="nf">html_attr</span><span class="p">(</span><span class="n">lists</span><span class="p">,</span> <span class="s">&#34;href&#34;</span><span class="p">),</span>
</span></span><span class="line"><span class="cl">        <span class="n">title</span><span class="o">=</span><span class="nf">str_trim</span><span class="p">(</span><span class="nf">html_text</span><span class="p">(</span><span class="n">lists</span><span class="p">)),</span>
</span></span><span class="line"><span class="cl">        <span class="n">dop</span><span class="o">=</span><span class="nf">html_text</span><span class="p">(</span><span class="n">dop</span><span class="p">),</span>
</span></span><span class="line"><span class="cl">        <span class="n">stringsAsFactors</span><span class="o">=</span><span class="kc">FALSE</span><span class="p">))</span>
</span></span><span class="line"><span class="cl"><span class="p">}</span>
</span></span><span class="line"><span class="cl"><span class="c1"># 遍历urls，循环运行getTOC</span>
</span></span><span class="line"><span class="cl"><span class="n">toc</span> <span class="o">&lt;-</span> <span class="nf">lapply</span><span class="p">(</span><span class="n">urls</span><span class="p">,</span> <span class="n">getTOC</span><span class="p">)</span>
</span></span><span class="line"><span class="cl"><span class="nf">library</span><span class="p">(</span><span class="n">dplyr</span><span class="p">)</span>
</span></span><span class="line"><span class="cl"><span class="c1"># 将23个数据框列表合并起来</span>
</span></span><span class="line"><span class="cl"><span class="n">toc</span> <span class="o">&lt;-</span> <span class="nf">do.call</span><span class="p">(</span><span class="s">&#34;bind_rows&#34;</span><span class="p">,</span> <span class="n">toc</span><span class="p">)</span>
</span></span></code></pre></td></tr></table>
</div>
</div><p>当然，这些动态不都是疫情报告。所以还要利用正则表达式过滤一下，并把链接格式补完。</p>
<ul>
<li>翻了几页，所有月度疫情公报都有“xx月全国法定”或“xx月份全国法定”的字样。</li>
<li>所有链接都是相对路径，形如&quot;../../xxx&quot;，需要替换为完整路径。</li>
</ul>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span><span class="lnt">5
</span><span class="lnt">6
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-r" data-lang="r"><span class="line"><span class="cl"><span class="nf">library</span><span class="p">(</span><span class="n">stringr</span><span class="p">)</span>
</span></span><span class="line"><span class="cl"><span class="c1"># 过滤出月报标题</span>
</span></span><span class="line"><span class="cl"><span class="n">toc</span> <span class="o">&lt;-</span> <span class="n">toc</span><span class="nf">[str_detect</span><span class="p">(</span><span class="n">toc</span><span class="o">$</span><span class="n">title</span><span class="p">,</span> <span class="s">&#34;月[份]*全国法定&#34;</span><span class="p">),</span><span class="n">]</span>
</span></span><span class="line"><span class="cl"><span class="c1"># 补完链接路径</span>
</span></span><span class="line"><span class="cl"><span class="n">toc</span><span class="o">$</span><span class="n">href</span> <span class="o">&lt;-</span> <span class="nf">str_replace</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">    <span class="n">toc</span><span class="o">$</span><span class="n">href</span><span class="p">,</span> <span class="s">&#34;^.+/jkj(.+$)&#34;</span><span class="p">,</span> <span class="s">&#34;http://www.nhfpc.gov.cn/jkj\\1&#34;</span><span class="p">)</span>
</span></span></code></pre></td></tr></table>
</div>
</div><p>除了链接，我还想知道每份月报讲的是何年何月，将来解析了数据，直接可以对上日期。这时候标题就派上用场了，因为里面直接包含有年月信息。但这里就有一个坑：不是所有标题都完整。最早的一些月报是不含年份的，得结合发布日期来补全。</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-r" data-lang="r"><span class="line"><span class="cl"><span class="c1"># 哪些月报标题里没有“年”字？</span>
</span></span><span class="line"><span class="cl"><span class="n">idx.noyr</span> <span class="o">&lt;-</span> <span class="nf">which</span><span class="p">(</span><span class="o">!</span> <span class="nf">str_detect</span><span class="p">(</span><span class="n">toc</span><span class="o">$</span><span class="n">title</span><span class="p">,</span> <span class="s">&#34;\\d+年&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl"><span class="c1"># 生成date列，初始均为NA</span>
</span></span><span class="line"><span class="cl"><span class="n">toc</span><span class="o">$</span><span class="n">date</span> <span class="o">&lt;-</span> <span class="kc">NA</span>
</span></span><span class="line"><span class="cl"><span class="c1"># 没有“年”字的月报，就用发布日期的年份来替代</span>
</span></span><span class="line"><span class="cl"><span class="n">toc</span><span class="o">$</span><span class="n">date[idx.noyr]</span> <span class="o">&lt;-</span> <span class="nf">paste</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">    <span class="nf">str_replace</span><span class="p">(</span><span class="n">toc</span><span class="o">$</span><span class="n">dop[idx.noyr]</span><span class="p">,</span> <span class="s">&#34;(^\\d+).+&#34;</span><span class="p">,</span> <span class="s">&#34;\\1&#34;</span><span class="p">),</span>
</span></span><span class="line"><span class="cl">    <span class="nf">str_replace</span><span class="p">(</span><span class="n">toc</span><span class="o">$</span><span class="n">title[idx.noyr]</span><span class="p">,</span> <span class="s">&#34;.+(\\d+)月.+&#34;</span><span class="p">,</span> <span class="s">&#34;\\1&#34;</span><span class="p">),</span> <span class="s">&#34;1&#34;</span><span class="p">,</span>
</span></span><span class="line"><span class="cl">    <span class="n">sep</span><span class="o">=</span><span class="s">&#34;-&#34;</span><span class="p">)</span>
</span></span><span class="line"><span class="cl"><span class="c1"># 其他月报，就直接提取标题里的年月信息</span>
</span></span><span class="line"><span class="cl"><span class="n">toc</span><span class="o">$</span><span class="n">date[</span><span class="o">-</span><span class="n">idx.noyr]</span> <span class="o">&lt;-</span> <span class="nf">str_replace</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">    <span class="n">toc</span><span class="o">$</span><span class="n">title[</span><span class="o">-</span><span class="n">idx.noyr]</span><span class="p">,</span> <span class="s">&#34;\\D*(\\d+)年(\\d+)月.+&#34;</span><span class="p">,</span> <span class="s">&#34;\\1-\\2-1&#34;</span><span class="p">)</span>
</span></span><span class="line"><span class="cl"><span class="c1"># 默认都是当月1日。date列转为日期型</span>
</span></span><span class="line"><span class="cl"><span class="n">toc</span><span class="o">$</span><span class="n">date</span> <span class="o">&lt;-</span> <span class="nf">as.Date</span><span class="p">(</span><span class="n">toc</span><span class="o">$</span><span class="n">date</span><span class="p">)</span>
</span></span></code></pre></td></tr></table>
</div>
</div><p><code>toc</code>数据集长这样：</p>
<!-- {% raw %} -->
<figure class="center"><img src="https://gh-1251443721.cos.ap-chengdu.myqcloud.com/2017/0406/toc_tbl.png"/><figcaption>
            <h4>图 | toc表格</h4>
        </figcaption>
</figure>
<!-- {% endraw %} -->
<p>万里长征踏出了第一步。</p>
<h2 id="爬网页">爬网页</h2>
<p>这一步就比较简单了，干脆就先把网页代码先弄下来。在这里就不多考虑反爬虫问题了（因为试下来卫计委官网好像没有反爬虫机制，再说才爬它一百多个页面，有啥好反的）。电脑虽然配置不济，好歹有4个核。为了加点速，充分利用CPU（R默认只用单核）算力，不妨拿<code>doParallel</code>包做点并行计算处理。</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-r" data-lang="r"><span class="line"><span class="cl"><span class="c1"># 构造一个读取网页代码的函数</span>
</span></span><span class="line"><span class="cl"><span class="n">getWebPage</span> <span class="o">&lt;-</span> <span class="nf">function</span><span class="p">(</span><span class="n">url</span><span class="p">){</span>
</span></span><span class="line"><span class="cl">    <span class="c1">## Arg</span>
</span></span><span class="line"><span class="cl">    <span class="c1">##    url: 网页网址</span>
</span></span><span class="line"><span class="cl">    <span class="n">html</span> <span class="o">&lt;-</span> <span class="nf">read_lines</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
</span></span><span class="line"><span class="cl">    <span class="nf">return</span><span class="p">(</span><span class="nf">paste</span><span class="p">(</span><span class="n">html</span><span class="p">,</span> <span class="n">collapse</span><span class="o">=</span><span class="s">&#34;\n&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl"><span class="p">}</span>
</span></span><span class="line"><span class="cl"><span class="c1"># 并行计算</span>
</span></span><span class="line"><span class="cl"><span class="nf">library</span><span class="p">(</span><span class="n">doParallel</span><span class="p">)</span>
</span></span><span class="line"><span class="cl"><span class="nf">registerDoParallel</span><span class="p">(</span><span class="n">cores</span><span class="o">=</span><span class="n">parallel</span><span class="o">::</span><span class="nf">detectCores</span><span class="p">())</span>
</span></span><span class="line"><span class="cl"><span class="n">pages</span> <span class="o">&lt;-</span> <span class="nf">foreach</span><span class="p">(</span><span class="n">i</span><span class="o">=</span><span class="nf">seq_along</span><span class="p">(</span><span class="n">toc</span><span class="o">$</span><span class="n">href</span><span class="p">),</span> <span class="n">.combine</span><span class="o">=</span><span class="n">c</span><span class="p">)</span> <span class="o">%dopar%</span>
</span></span><span class="line"><span class="cl">    <span class="nf">invisible</span><span class="p">(</span><span class="nf">getWebPage</span><span class="p">(</span><span class="n">toc</span><span class="o">$</span><span class="n">href[i]</span><span class="p">))</span>
</span></span><span class="line"><span class="cl"><span class="nf">names</span><span class="p">(</span><span class="n">pages</span><span class="p">)</span> <span class="o">&lt;-</span> <span class="nf">as.character</span><span class="p">(</span><span class="n">toc</span><span class="o">$</span><span class="n">date</span><span class="p">)</span>
</span></span></code></pre></td></tr></table>
</div>
</div><p>pages是一个149个文本元素构成的大向量，用日期作为向量命名。</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-r" data-lang="r"><span class="line"><span class="cl"><span class="nf">str</span><span class="p">(</span><span class="n">pages</span><span class="p">)</span>
</span></span></code></pre></td></tr></table>
</div>
</div><div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-r" data-lang="r"><span class="line"><span class="cl"> <span class="n">Named</span> <span class="n">chr</span> <span class="n">[1</span><span class="o">:</span><span class="m">149</span><span class="n">]</span> <span class="s">&#34;  &lt;!DOCTYPE html PUBLIC \&#34;-//W3C//DTD XHTML 1.0 Transitional//EN\&#34; \&#34;http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd\&#34;&gt;&#34;</span><span class="o">|</span> <span class="n">__truncated__</span> <span class="kc">...</span>
</span></span><span class="line"><span class="cl"> <span class="o">-</span> <span class="nf">attr</span><span class="p">(</span><span class="o">*</span><span class="p">,</span> <span class="s">&#34;names&#34;</span><span class="p">)</span><span class="o">=</span> <span class="n">chr</span> <span class="n">[1</span><span class="o">:</span><span class="m">149</span><span class="n">]</span> <span class="s">&#34;2017-02-01&#34;</span> <span class="s">&#34;2017-01-01&#34;</span> <span class="s">&#34;2016-12-01&#34;</span> <span class="s">&#34;2016-11-01&#34;</span> <span class="kc">...</span>
</span></span></code></pre></td></tr></table>
</div>
</div><h2 id="解析发病和死亡总数">解析发病和死亡总数</h2>
<p>第二个大坑正在缓缓靠近：不是所有月报都有发病和死亡总数。</p>
<ul>
<li>2005年以前，压根不报告丙类传染病</li>
<li>2010年以前，不汇报发病和死亡总数，且甲乙类合并计数，丙类另报</li>
<li>甲类汇总的文本多样化很强，一般鼠疫和霍乱还分开来报，正则写起来会死。</li>
</ul>
<p>所以最后决定：</p>
<ul>
<li>抽得到总数的，直接用总数</li>
<li>没有报告总数的，会向后抽到甲乙类合计，或甲类，或乙类。无论哪种情况，乙类+丙类基本等于总数</li>
</ul>
<p>好机（偷）智（懒）！</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span><span class="lnt">27
</span><span class="lnt">28
</span><span class="lnt">29
</span><span class="lnt">30
</span><span class="lnt">31
</span><span class="lnt">32
</span><span class="lnt">33
</span><span class="lnt">34
</span><span class="lnt">35
</span><span class="lnt">36
</span><span class="lnt">37
</span><span class="lnt">38
</span><span class="lnt">39
</span><span class="lnt">40
</span><span class="lnt">41
</span><span class="lnt">42
</span><span class="lnt">43
</span><span class="lnt">44
</span><span class="lnt">45
</span><span class="lnt">46
</span><span class="lnt">47
</span><span class="lnt">48
</span><span class="lnt">49
</span><span class="lnt">50
</span><span class="lnt">51
</span><span class="lnt">52
</span><span class="lnt">53
</span><span class="lnt">54
</span><span class="lnt">55
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-r" data-lang="r"><span class="line"><span class="cl"><span class="c1"># 构造函数，从正文直接提取发病和死亡总数</span>
</span></span><span class="line"><span class="cl"><span class="n">getKeyNums</span> <span class="o">&lt;-</span> <span class="nf">function</span><span class="p">(</span><span class="n">page</span><span class="p">){</span>
</span></span><span class="line"><span class="cl">    <span class="c1">## Arg</span>
</span></span><span class="line"><span class="cl">    <span class="c1">##   page: 单个网页</span>
</span></span><span class="line"><span class="cl">
</span></span><span class="line"><span class="cl">    <span class="n">page</span> <span class="o">&lt;-</span> <span class="nf">read_html</span><span class="p">(</span><span class="n">page</span><span class="p">)</span>
</span></span><span class="line"><span class="cl">    <span class="n">txt.node</span> <span class="o">&lt;-</span> <span class="nf">html_nodes</span><span class="p">(</span><span class="n">page</span><span class="p">,</span> <span class="s">&#34;div.con&#34;</span><span class="p">)</span>
</span></span><span class="line"><span class="cl">    <span class="n">txt</span> <span class="o">&lt;-</span> <span class="nf">html_text</span><span class="p">(</span><span class="n">txt.node</span><span class="p">)</span>  <span class="c1"># 获得月报正文</span>
</span></span><span class="line"><span class="cl">    <span class="c1">## 发病总数</span>
</span></span><span class="line"><span class="cl">    <span class="n">inc.tot</span> <span class="o">&lt;-</span> <span class="nf">as.integer</span><span class="p">(</span><span class="nf">str_replace</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">        <span class="n">txt</span><span class="p">,</span> <span class="nf">regex</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">            <span class="s">&#34;.+?(发病|报告)\\D+(\\d{6,})[例人].+&#34;</span><span class="p">,</span> <span class="n">dotall</span><span class="o">=</span><span class="kc">TRUE</span>
</span></span><span class="line"><span class="cl">            <span class="p">),</span> <span class="s">&#34;\\2&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">    <span class="c1">## 死亡总数</span>
</span></span><span class="line"><span class="cl">    <span class="n">mot.tot</span><span class="o">&lt;-</span> <span class="nf">as.integer</span><span class="p">(</span><span class="nf">str_replace</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">        <span class="n">txt</span><span class="p">,</span> <span class="nf">regex</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">            <span class="s">&#34;.+?(发病|报告).+?死亡\\D*?(\\d+)[例人].+&#34;</span><span class="p">,</span> <span class="n">dotall</span><span class="o">=</span><span class="kc">TRUE</span>
</span></span><span class="line"><span class="cl">            <span class="p">),</span> <span class="s">&#34;\\2&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">    <span class="c1">## 乙类发病总数</span>
</span></span><span class="line"><span class="cl">    <span class="n">inc.b</span> <span class="o">&lt;-</span> <span class="nf">as.integer</span><span class="p">(</span><span class="nf">str_replace</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">        <span class="n">txt</span><span class="p">,</span> <span class="nf">regex</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">            <span class="s">&#34;.+?乙类.+?(\\d+)[例人].+&#34;</span><span class="p">,</span> <span class="n">dotall</span><span class="o">=</span><span class="kc">TRUE</span>
</span></span><span class="line"><span class="cl">            <span class="p">),</span> <span class="s">&#34;\\1&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">    <span class="c1">## 乙类死亡总数</span>
</span></span><span class="line"><span class="cl">    <span class="n">mot.b</span> <span class="o">&lt;-</span> <span class="nf">as.integer</span><span class="p">(</span><span class="nf">str_replace</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">        <span class="n">txt</span><span class="p">,</span> <span class="nf">regex</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">            <span class="s">&#34;.+?乙类.+?死亡\\D*?(\\d+)[例人].+&#34;</span><span class="p">,</span> <span class="n">dotall</span><span class="o">=</span><span class="kc">TRUE</span><span class="p">),</span>
</span></span><span class="line"><span class="cl">        <span class="s">&#34;\\1&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">    <span class="c1">## 丙类发病总数</span>
</span></span><span class="line"><span class="cl">    <span class="n">inc.c</span> <span class="o">&lt;-</span> <span class="nf">as.integer</span><span class="p">(</span><span class="nf">str_replace</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">        <span class="n">txt</span><span class="p">,</span> <span class="nf">regex</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">            <span class="s">&#34;.+?丙类.+?(\\d+)[例人].+&#34;</span><span class="p">,</span> <span class="n">dotall</span><span class="o">=</span><span class="kc">TRUE</span>
</span></span><span class="line"><span class="cl">            <span class="p">),</span> <span class="s">&#34;\\1&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">    <span class="c1">## 丙类死亡总数</span>
</span></span><span class="line"><span class="cl">    <span class="n">mot.c</span> <span class="o">&lt;-</span> <span class="nf">as.integer</span><span class="p">(</span><span class="nf">str_replace</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">        <span class="n">txt</span><span class="p">,</span> <span class="nf">regex</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">            <span class="s">&#34;.+?丙类.+?死亡\\D*?(\\d+)[例人].+&#34;</span><span class="p">,</span> <span class="n">dotall</span><span class="o">=</span><span class="kc">TRUE</span>
</span></span><span class="line"><span class="cl">            <span class="p">),</span> <span class="s">&#34;\\1&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">    <span class="c1">## 如发病总数=乙类发病数，则未提取到，用乙类+丙类替代</span>
</span></span><span class="line"><span class="cl">    <span class="nf">if </span><span class="p">(</span><span class="nf">identical</span><span class="p">(</span><span class="n">inc.tot</span><span class="p">,</span> <span class="n">inc.b</span><span class="p">))</span> <span class="n">inc.tot</span> <span class="o">&lt;-</span> <span class="n">inc.tot</span> <span class="o">+</span> <span class="n">inc.c</span>
</span></span><span class="line"><span class="cl">    <span class="nf">if </span><span class="p">(</span><span class="nf">identical</span><span class="p">(</span><span class="n">mot.tot</span><span class="p">,</span> <span class="n">mot.b</span><span class="p">))</span> <span class="n">mot.tot</span> <span class="o">&lt;-</span> <span class="n">mot.tot</span> <span class="o">+</span> <span class="n">mot.c</span>
</span></span><span class="line"><span class="cl">    <span class="c1">## 返回发病总数和死亡总数</span>
</span></span><span class="line"><span class="cl">    <span class="nf">return</span><span class="p">(</span><span class="nf">c</span><span class="p">(</span><span class="n">inc.tot</span><span class="p">,</span> <span class="n">mot.tot</span><span class="p">))</span>
</span></span><span class="line"><span class="cl"><span class="p">}</span>
</span></span><span class="line"><span class="cl"><span class="c1"># 遍历pages运行getKeyNums，转置后转换为2列的数据框</span>
</span></span><span class="line"><span class="cl"><span class="n">genl.stat</span> <span class="o">&lt;-</span> <span class="nf">as.data.frame</span><span class="p">(</span><span class="nf">t</span><span class="p">(</span><span class="nf">sapply</span><span class="p">(</span><span class="n">pages</span><span class="p">,</span> <span class="n">getKeyNums</span><span class="p">)))</span>
</span></span><span class="line"><span class="cl"><span class="nf">names</span><span class="p">(</span><span class="n">genl.stat</span><span class="p">)</span> <span class="o">&lt;-</span> <span class="nf">c</span><span class="p">(</span><span class="s">&#34;Incidence&#34;</span><span class="p">,</span> <span class="s">&#34;Mortality&#34;</span><span class="p">)</span>
</span></span><span class="line"><span class="cl"><span class="c1"># 将row.names衍生为新变量Date，转为日期型</span>
</span></span><span class="line"><span class="cl"><span class="n">genl.stat</span><span class="o">$</span><span class="n">Date</span> <span class="o">&lt;-</span> <span class="nf">as.Date</span><span class="p">(</span><span class="nf">row.names</span><span class="p">(</span><span class="n">genl.stat</span><span class="p">))</span>
</span></span><span class="line"><span class="cl"><span class="c1"># 有重复月报！去掉重复记录。取2005年以后的月报</span>
</span></span><span class="line"><span class="cl"><span class="n">genl.stat</span> <span class="o">&lt;-</span> <span class="n">genl.stat[</span>
</span></span><span class="line"><span class="cl">     <span class="o">!</span> <span class="nf">duplicated</span><span class="p">(</span><span class="n">genl.stat</span><span class="o">$</span><span class="n">Date</span><span class="p">)</span> <span class="o">&amp;</span> <span class="n">genl.stat</span><span class="o">$</span><span class="n">Date</span> <span class="o">&gt;</span> <span class="s">&#34;2004-12-01&#34;</span><span class="p">,</span><span class="n">]</span>
</span></span><span class="line"><span class="cl"><span class="n">genl.stat</span> <span class="o">&lt;-</span> <span class="n">genl.stat</span><span class="nf">[order</span><span class="p">(</span><span class="n">genl.stat</span><span class="o">$</span><span class="n">Date</span><span class="p">),</span><span class="n">]</span>  <span class="c1"># 按日期排序</span>
</span></span><span class="line"><span class="cl"><span class="c1"># 2009/4的月报无论如何不适配，只能手动改</span>
</span></span><span class="line"><span class="cl"><span class="n">genl.stat[</span><span class="s">&#34;2009-04-01&#34;</span><span class="p">,</span> <span class="m">1</span><span class="o">:</span><span class="m">2</span><span class="n">]</span> <span class="o">&lt;-</span> <span class="nf">c</span><span class="p">(</span><span class="m">338281</span><span class="p">,</span> <span class="m">576</span><span class="p">)</span>
</span></span></code></pre></td></tr></table>
</div>
</div><p>历尽艰苦，终于得到了这个该死的数据集。把历月发病和死亡数做个时序图。</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span><span class="lnt">3
</span><span class="lnt">4
</span><span class="lnt">5
</span><span class="lnt">6
</span><span class="lnt">7
</span><span class="lnt">8
</span><span class="lnt">9
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-r" data-lang="r"><span class="line"><span class="cl"><span class="n">genl.stat</span> <span class="o">&lt;-</span> <span class="nf">melt</span><span class="p">(</span><span class="n">genl.stat</span><span class="p">,</span> <span class="n">id</span><span class="o">=</span><span class="s">&#34;Date&#34;</span><span class="p">)</span>
</span></span><span class="line"><span class="cl"><span class="nf">library</span><span class="p">(</span><span class="n">ggplot2</span><span class="p">)</span>
</span></span><span class="line"><span class="cl"><span class="nf">library</span><span class="p">(</span><span class="n">ggthemes</span><span class="p">)</span>
</span></span><span class="line"><span class="cl"><span class="nf">ggplot</span><span class="p">(</span><span class="n">genl.stat</span><span class="p">)</span> <span class="o">+</span> <span class="nf">geom_line</span><span class="p">(</span><span class="nf">aes</span><span class="p">(</span><span class="n">Date</span><span class="p">,</span> <span class="n">value</span><span class="p">,</span> <span class="n">color</span><span class="o">=</span><span class="n">variable</span><span class="p">))</span> <span class="o">+</span> <span class="nf">theme_hc</span><span class="p">()</span> <span class="o">+</span>
</span></span><span class="line"><span class="cl">    <span class="nf">scale_color_hc</span><span class="p">()</span> <span class="o">+</span> <span class="nf">scale_x_date</span><span class="p">(</span><span class="n">date_breaks</span><span class="o">=</span><span class="s">&#34;1 year&#34;</span><span class="p">,</span> <span class="n">date_labels</span><span class="o">=</span><span class="s">&#34;%Y&#34;</span><span class="p">)</span> <span class="o">+</span>
</span></span><span class="line"><span class="cl">    <span class="nf">facet_grid</span><span class="p">(</span><span class="n">variable</span><span class="o">~</span><span class="n">.,</span> <span class="n">scales</span><span class="o">=</span><span class="s">&#34;free&#34;</span><span class="p">)</span> <span class="o">+</span>
</span></span><span class="line"><span class="cl">    <span class="nf">theme</span><span class="p">(</span><span class="n">axis.ticks</span><span class="o">=</span><span class="nf">element_line</span><span class="p">(</span><span class="n">linetype</span><span class="o">=</span><span class="m">0</span><span class="p">))</span> <span class="o">+</span>
</span></span><span class="line"><span class="cl">    <span class="nf">labs</span><span class="p">(</span><span class="n">title</span><span class="o">=</span><span class="s">&#34;Incidence And Mortality of Notifiable Infectious Diseases&#34;</span><span class="p">,</span>
</span></span><span class="line"><span class="cl">        <span class="n">subtitle</span><span class="o">=</span><span class="s">&#34;2005/1-2017/2&#34;</span><span class="p">,</span> <span class="n">caption</span><span class="o">=</span><span class="s">&#34;source: NHFPC&#34;</span><span class="p">)</span>
</span></span></code></pre></td></tr></table>
</div>
</div><p>年周期性还是很显著的。</p>
<!-- {% raw %} -->
<figure class="center"><img src="https://gh-1251443721.cos.ap-chengdu.myqcloud.com/2017/0406/inc_mot_infectdis.png"/><figcaption>
            <h4>图 | 发病/死亡月度统计</h4>
        </figcaption>
</figure>
<!-- {% endraw %} -->
<h2 id="抽取各明细病种数据">抽取各明细病种数据</h2>
<p>接下来进行明细分病种数据的爬取。</p>
<p>理论上，把表格抽取出来就完了嘛。但是尝试了一下发现竟然有问题。随机看了一下，发现自己被坑惨了。附表有三类：</p>
<ul>
<li>网页表格，这种最好利用</li>
<li>附件，如doc、xls</li>
<li>图片（！），如png，jpg</li>
</ul>
<p>竟然还有直接传个截图当公报的，怎么不去爆炸？</p>
<p>本来打算<code>html_table</code>跑一遍就愉快地合并数据框了，却不料跑进了一个马里亚纳深坑里。只能改变策略，先把这些附件都下载下来。</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt"> 1
</span><span class="lnt"> 2
</span><span class="lnt"> 3
</span><span class="lnt"> 4
</span><span class="lnt"> 5
</span><span class="lnt"> 6
</span><span class="lnt"> 7
</span><span class="lnt"> 8
</span><span class="lnt"> 9
</span><span class="lnt">10
</span><span class="lnt">11
</span><span class="lnt">12
</span><span class="lnt">13
</span><span class="lnt">14
</span><span class="lnt">15
</span><span class="lnt">16
</span><span class="lnt">17
</span><span class="lnt">18
</span><span class="lnt">19
</span><span class="lnt">20
</span><span class="lnt">21
</span><span class="lnt">22
</span><span class="lnt">23
</span><span class="lnt">24
</span><span class="lnt">25
</span><span class="lnt">26
</span><span class="lnt">27
</span><span class="lnt">28
</span><span class="lnt">29
</span><span class="lnt">30
</span><span class="lnt">31
</span><span class="lnt">32
</span><span class="lnt">33
</span><span class="lnt">34
</span><span class="lnt">35
</span><span class="lnt">36
</span><span class="lnt">37
</span><span class="lnt">38
</span><span class="lnt">39
</span><span class="lnt">40
</span><span class="lnt">41
</span><span class="lnt">42
</span><span class="lnt">43
</span><span class="lnt">44
</span><span class="lnt">45
</span><span class="lnt">46
</span><span class="lnt">47
</span><span class="lnt">48
</span><span class="lnt">49
</span><span class="lnt">50
</span><span class="lnt">51
</span><span class="lnt">52
</span><span class="lnt">53
</span><span class="lnt">54
</span><span class="lnt">55
</span><span class="lnt">56
</span><span class="lnt">57
</span><span class="lnt">58
</span><span class="lnt">59
</span><span class="lnt">60
</span><span class="lnt">61
</span><span class="lnt">62
</span><span class="lnt">63
</span><span class="lnt">64
</span><span class="lnt">65
</span><span class="lnt">66
</span><span class="lnt">67
</span><span class="lnt">68
</span><span class="lnt">69
</span><span class="lnt">70
</span><span class="lnt">71
</span><span class="lnt">72
</span><span class="lnt">73
</span><span class="lnt">74
</span><span class="lnt">75
</span><span class="lnt">76
</span><span class="lnt">77
</span><span class="lnt">78
</span><span class="lnt">79
</span><span class="lnt">80
</span><span class="lnt">81
</span><span class="lnt">82
</span><span class="lnt">83
</span><span class="lnt">84
</span><span class="lnt">85
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-r" data-lang="r"><span class="line"><span class="cl"><span class="c1"># 构造函数，下载附表</span>
</span></span><span class="line"><span class="cl"><span class="n">getWebTbl</span> <span class="o">&lt;-</span> <span class="nf">function</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="n">tbl.name</span><span class="p">){</span>
</span></span><span class="line"><span class="cl">    <span class="c1">## Arg</span>
</span></span><span class="line"><span class="cl">    <span class="c1">##    url: 网页</span>
</span></span><span class="line"><span class="cl">    <span class="c1">##    tbl.name: 表格名称，用日期命名</span>
</span></span><span class="line"><span class="cl">
</span></span><span class="line"><span class="cl">    <span class="c1"># 如附表文件已存在，跳出</span>
</span></span><span class="line"><span class="cl">    <span class="nf">if </span><span class="p">(</span><span class="nf">any</span><span class="p">(</span><span class="nf">file.exists</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">        <span class="nf">paste0</span><span class="p">(</span><span class="s">&#34;~/infectdis/&#34;</span><span class="p">,</span> <span class="n">tbl.name</span><span class="p">,</span> <span class="s">&#34;.&#34;</span><span class="p">,</span>
</span></span><span class="line"><span class="cl">               <span class="nf">c</span><span class="p">(</span><span class="s">&#34;xls&#34;</span><span class="p">,</span> <span class="s">&#34;csv&#34;</span><span class="p">,</span> <span class="s">&#34;doc&#34;</span><span class="p">,</span> <span class="s">&#34;gif&#34;</span><span class="p">,</span> <span class="s">&#34;jpg&#34;</span><span class="p">,</span> <span class="s">&#34;png&#34;</span><span class="p">))))){</span>
</span></span><span class="line"><span class="cl">        <span class="nf">return</span><span class="p">(</span><span class="nf">invisible</span><span class="p">())</span>
</span></span><span class="line"><span class="cl">    <span class="p">}</span>
</span></span><span class="line"><span class="cl">    <span class="c1"># 否则就把页面源码读下来</span>
</span></span><span class="line"><span class="cl">    <span class="n">html</span> <span class="o">&lt;-</span> <span class="nf">read_html</span><span class="p">(</span><span class="n">url</span><span class="p">)</span>
</span></span><span class="line"><span class="cl">    <span class="c1"># 尝试抽取网页表格</span>
</span></span><span class="line"><span class="cl">    <span class="n">cast</span> <span class="o">&lt;-</span> <span class="nf">html_nodes</span><span class="p">(</span><span class="n">html</span><span class="p">,</span> <span class="s">&#34;table&#34;</span><span class="p">)</span>
</span></span><span class="line"><span class="cl">    <span class="c1"># 尝试抽取附件表格</span>
</span></span><span class="line"><span class="cl">    <span class="n">cast.attach</span> <span class="o">&lt;-</span> <span class="nf">html_nodes</span><span class="p">(</span><span class="n">html</span><span class="p">,</span> <span class="s">&#34;div.con a&#34;</span><span class="p">)</span>
</span></span><span class="line"><span class="cl">    <span class="n">regex.attach</span> <span class="o">&lt;-</span> <span class="s">&#34;([Xx][Ll][Ss][Xx]*|[Dd][Oo][Cc][Xx]*)&#34;</span>
</span></span><span class="line"><span class="cl">    <span class="c1"># 尝试抽取附件图片</span>
</span></span><span class="line"><span class="cl">    <span class="n">cast.img</span> <span class="o">&lt;-</span> <span class="nf">html_nodes</span><span class="p">(</span><span class="n">html</span><span class="p">,</span> <span class="s">&#34;div.con img&#34;</span><span class="p">)</span>
</span></span><span class="line"><span class="cl">    <span class="n">regex.img</span> <span class="o">&lt;-</span> <span class="s">&#34;([Gg][Ii][Ff]|[Pp][Nn][Gg]|[Jj][Pp][Gg])&#34;</span>
</span></span><span class="line"><span class="cl">    <span class="nf">if </span><span class="p">(</span><span class="nf">length</span><span class="p">(</span><span class="n">cast</span><span class="p">)</span><span class="o">&gt;</span><span class="m">0</span><span class="p">){</span>  <span class="c1"># 读到网页表格</span>
</span></span><span class="line"><span class="cl">        <span class="n">out</span> <span class="o">&lt;-</span> <span class="nf">html_table</span><span class="p">(</span><span class="n">cast</span><span class="p">,</span> <span class="n">fill</span><span class="o">=</span><span class="kc">TRUE</span><span class="p">)</span><span class="n">[[1]]</span>
</span></span><span class="line"><span class="cl">        <span class="nf">if </span><span class="p">(</span><span class="o">!</span> <span class="nf">file.exists</span><span class="p">(</span><span class="nf">paste0</span><span class="p">(</span><span class="s">&#34;~/infectdis/&#34;</span><span class="p">,</span> <span class="n">tbl.name</span><span class="p">,</span> <span class="s">&#34;.csv&#34;</span><span class="p">))){</span>
</span></span><span class="line"><span class="cl">            <span class="nf">write_csv</span><span class="p">(</span><span class="n">out</span><span class="p">,</span> <span class="nf">paste0</span><span class="p">(</span><span class="s">&#34;~/infectdis/&#34;</span><span class="p">,</span> <span class="n">tbl.name</span><span class="p">,</span> <span class="s">&#34;.csv&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">        <span class="p">}</span>
</span></span><span class="line"><span class="cl">    <span class="p">}</span> <span class="n">else</span> <span class="nf">if </span><span class="p">(</span><span class="nf">any</span><span class="p">(</span><span class="nf">str_detect</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">        <span class="n">cast.attach</span><span class="p">,</span> <span class="nf">paste0</span><span class="p">(</span><span class="s">&#34;\\.&#34;</span><span class="p">,</span> <span class="n">regex.attach</span><span class="p">,</span> <span class="s">&#34;\&#34;&#34;</span><span class="p">)))){</span>  
</span></span><span class="line"><span class="cl">        <span class="c1"># 读到附件表格</span>
</span></span><span class="line"><span class="cl">        <span class="n">idx.attach</span> <span class="o">&lt;-</span> <span class="nf">which</span><span class="p">(</span><span class="nf">str_detect</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">            <span class="n">cast.attach</span><span class="p">,</span> <span class="nf">paste0</span><span class="p">(</span><span class="s">&#34;\\.&#34;</span><span class="p">,</span> <span class="n">regex.attach</span><span class="p">,</span> <span class="s">&#34;\&#34;&#34;</span><span class="p">)))</span><span class="n">[1]</span>
</span></span><span class="line"><span class="cl">        <span class="n">doc.link</span> <span class="o">&lt;-</span> <span class="nf">str_replace</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">            <span class="n">cast.attach[idx.attach]</span><span class="p">,</span>
</span></span><span class="line"><span class="cl">            <span class="nf">paste0</span><span class="p">(</span><span class="s">&#34;.+href=\&#34;(.+?\\.)&#34;</span><span class="p">,</span> <span class="n">regex.attach</span><span class="p">,</span> <span class="s">&#34;\&#34;.+&#34;</span><span class="p">),</span> <span class="s">&#34;\\1\\2&#34;</span><span class="p">)</span>
</span></span><span class="line"><span class="cl">        <span class="n">file.type</span> <span class="o">&lt;-</span> <span class="nf">tolower</span><span class="p">(</span><span class="nf">str_replace</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">            <span class="n">doc.link</span><span class="p">,</span> <span class="nf">paste0</span><span class="p">(</span><span class="s">&#34;.+\\.&#34;</span><span class="p">,</span> <span class="n">regex.attach</span><span class="p">,</span> <span class="s">&#34;$&#34;</span><span class="p">),</span> <span class="s">&#34;\\1&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">        <span class="c1"># 附件路径补完</span>
</span></span><span class="line"><span class="cl">        <span class="nf">if </span><span class="p">(</span><span class="nf">str_detect</span><span class="p">(</span><span class="n">doc.link</span><span class="p">,</span> <span class="s">&#34;^/&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">            <span class="n">doc.link</span> <span class="o">&lt;-</span> <span class="nf">paste0</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">                <span class="s">&#34;http://www.nhfpc.gov.cn&#34;</span><span class="p">,</span> <span class="n">doc.link</span><span class="p">)</span>
</span></span><span class="line"><span class="cl">        <span class="c1"># 另一种类型的相对路径</span>
</span></span><span class="line"><span class="cl">        <span class="nf">if </span><span class="p">(</span><span class="nf">str_detect</span><span class="p">(</span><span class="n">doc.link</span><span class="p">,</span> <span class="s">&#34;^[^h/]&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">            <span class="n">doc.link</span> <span class="o">&lt;-</span> <span class="nf">paste0</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">                <span class="nf">str_replace</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="s">&#34;^(.+)\\.shtml$&#34;</span><span class="p">,</span> <span class="s">&#34;\\1&#34;</span><span class="p">),</span>
</span></span><span class="line"><span class="cl">                <span class="nf">str_replace</span><span class="p">(</span><span class="n">doc.link</span><span class="p">,</span> <span class="s">&#34;^[^/]+(/.+$)&#34;</span><span class="p">,</span> <span class="s">&#34;\\1&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">        <span class="c1"># 重命名并存储</span>
</span></span><span class="line"><span class="cl">        <span class="nf">if </span><span class="p">(</span><span class="o">!</span> <span class="nf">file.exists</span><span class="p">(</span><span class="nf">paste0</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">            <span class="s">&#34;~/infectdis/&#34;</span><span class="p">,</span> <span class="n">tbl.name</span><span class="p">,</span> <span class="s">&#34;.&#34;</span><span class="p">,</span> <span class="n">file.type</span><span class="p">))){</span>
</span></span><span class="line"><span class="cl">            <span class="n">doc.file</span> <span class="o">&lt;-</span> <span class="nf">download.file</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">                <span class="n">doc.link</span><span class="p">,</span> <span class="n">destfile</span><span class="o">=</span><span class="nf">paste0</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">                    <span class="s">&#34;~/infectdis/&#34;</span><span class="p">,</span> <span class="n">tbl.name</span><span class="p">,</span> <span class="s">&#34;.&#34;</span><span class="p">,</span> <span class="n">file.type</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">        <span class="p">}</span>
</span></span><span class="line"><span class="cl">    <span class="p">}</span> <span class="n">else</span> <span class="nf">if </span><span class="p">(</span><span class="nf">any</span><span class="p">(</span><span class="nf">str_detect</span><span class="p">(</span><span class="n">cast.img</span><span class="p">,</span> <span class="nf">paste0</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">        <span class="s">&#34;\\.&#34;</span><span class="p">,</span> <span class="n">regex.img</span><span class="p">,</span> <span class="s">&#34;\&#34;&#34;</span><span class="p">)))){</span>
</span></span><span class="line"><span class="cl">        <span class="c1"># 读到附件图片</span>
</span></span><span class="line"><span class="cl">        <span class="n">idx.img</span> <span class="o">&lt;-</span> <span class="nf">which</span><span class="p">(</span><span class="nf">str_detect</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">            <span class="n">cast.img</span><span class="p">,</span> <span class="nf">paste0</span><span class="p">(</span><span class="s">&#34;\\.&#34;</span><span class="p">,</span> <span class="n">regex.img</span><span class="p">,</span> <span class="s">&#34;\&#34;&#34;</span><span class="p">)))</span><span class="n">[1]</span>
</span></span><span class="line"><span class="cl">        <span class="n">doc.link</span> <span class="o">&lt;-</span> <span class="nf">str_replace</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">            <span class="n">cast.img[idx.img]</span><span class="p">,</span>
</span></span><span class="line"><span class="cl">            <span class="nf">paste0</span><span class="p">(</span><span class="s">&#34;.+img.+src=\&#34;(.+?\\.)&#34;</span><span class="p">,</span> <span class="n">regex.img</span><span class="p">,</span> <span class="s">&#34;\&#34;.+&#34;</span><span class="p">),</span> <span class="s">&#34;\\1\\2&#34;</span><span class="p">)</span>
</span></span><span class="line"><span class="cl">        <span class="n">file.type</span> <span class="o">&lt;-</span> <span class="nf">tolower</span><span class="p">(</span><span class="nf">str_replace</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">            <span class="n">doc.link</span><span class="p">,</span> <span class="nf">paste0</span><span class="p">(</span><span class="s">&#34;.+\\.&#34;</span><span class="p">,</span> <span class="n">regex.img</span><span class="p">,</span> <span class="s">&#34;$&#34;</span><span class="p">),</span> <span class="s">&#34;\\1&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">        <span class="c1"># 附件路径补完        </span>
</span></span><span class="line"><span class="cl">        <span class="nf">if </span><span class="p">(</span><span class="nf">str_detect</span><span class="p">(</span><span class="n">doc.link</span><span class="p">,</span> <span class="s">&#34;^/&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">            <span class="n">doc.link</span> <span class="o">&lt;-</span> <span class="nf">paste0</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">                <span class="s">&#34;http://www.nhfpc.gov.cn&#34;</span><span class="p">,</span> <span class="n">doc.link</span><span class="p">)</span>
</span></span><span class="line"><span class="cl">        <span class="c1"># 另一种类型的相对路径</span>
</span></span><span class="line"><span class="cl">        <span class="nf">if </span><span class="p">(</span><span class="nf">str_detect</span><span class="p">(</span><span class="n">doc.link</span><span class="p">,</span> <span class="s">&#34;^[^h/]&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">            <span class="n">doc.link</span> <span class="o">&lt;-</span> <span class="nf">paste0</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">                <span class="nf">str_replace</span><span class="p">(</span><span class="n">url</span><span class="p">,</span> <span class="s">&#34;^(.+)\\.shtml$&#34;</span><span class="p">,</span> <span class="s">&#34;\\1&#34;</span><span class="p">),</span>
</span></span><span class="line"><span class="cl">                <span class="nf">str_replace</span><span class="p">(</span><span class="n">doc.link</span><span class="p">,</span> <span class="s">&#34;^[^/]+(/.+$)&#34;</span><span class="p">,</span> <span class="s">&#34;\\1&#34;</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">        <span class="c1"># 重命名并存储</span>
</span></span><span class="line"><span class="cl">        <span class="nf">if </span><span class="p">(</span><span class="o">!</span> <span class="nf">file.exists</span><span class="p">(</span><span class="nf">paste0</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">            <span class="s">&#34;~/infectdis/&#34;</span><span class="p">,</span> <span class="n">tbl.name</span><span class="p">,</span> <span class="s">&#34;.&#34;</span><span class="p">,</span> <span class="n">file.type</span><span class="p">))){</span>
</span></span><span class="line"><span class="cl">            <span class="n">doc.file</span> <span class="o">&lt;-</span> <span class="nf">download.file</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">                <span class="n">doc.link</span><span class="p">,</span> <span class="n">destfile</span><span class="o">=</span><span class="nf">paste0</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">                    <span class="s">&#34;~/infectdis/&#34;</span><span class="p">,</span> <span class="n">tbl.name</span><span class="p">,</span> <span class="s">&#34;.&#34;</span><span class="p">,</span> <span class="n">file.type</span><span class="p">))</span>
</span></span><span class="line"><span class="cl">        <span class="p">}</span>
</span></span><span class="line"><span class="cl">    <span class="p">}</span>
</span></span><span class="line"><span class="cl"><span class="p">}</span>
</span></span><span class="line"><span class="cl"><span class="c1"># 再次动用并行计算</span>
</span></span><span class="line"><span class="cl"><span class="nf">registerDoParallel</span><span class="p">(</span><span class="n">cores</span><span class="o">=</span><span class="n">parallel</span><span class="o">::</span><span class="nf">detectCores</span><span class="p">())</span>
</span></span><span class="line"><span class="cl"><span class="nf">foreach</span><span class="p">(</span><span class="n">i</span><span class="o">=</span><span class="nf">seq_along</span><span class="p">(</span><span class="n">toc</span><span class="o">$</span><span class="n">href</span><span class="p">))</span> <span class="o">%dopar%</span>
</span></span><span class="line"><span class="cl">    <span class="nf">invisible</span><span class="p">(</span><span class="nf">getWebTbl</span><span class="p">(</span><span class="n">toc</span><span class="o">$</span><span class="n">href[i]</span><span class="p">,</span> <span class="nf">as.character</span><span class="p">(</span><span class="n">toc</span><span class="o">$</span><span class="n">date[i]</span><span class="p">)))</span>
</span></span></code></pre></td></tr></table>
</div>
</div><p>看到这些文件齐齐整整码在硬盘里，心下暂时宽慰了一点。</p>
<!-- {% raw %} -->
<figure class="center"><img src="https://gh-1251443721.cos.ap-chengdu.myqcloud.com/2017/0406/downfiles.png"/><figcaption>
            <h4>图 | 附件下载</h4>
        </figcaption>
</figure>
<!-- {% endraw %} -->
<p>然而让我们回头看看疫情月报附表们可恨的多样性吧。</p>
<div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-r" data-lang="r"><span class="line"><span class="cl"><span class="nf">table</span><span class="p">(</span><span class="nf">str_replace</span><span class="p">(</span>
</span></span><span class="line"><span class="cl">    <span class="nf">list.files</span><span class="p">(</span><span class="s">&#34;~/infectdis&#34;</span><span class="p">),</span> <span class="s">&#34;.+\\.(.+$)&#34;</span><span class="p">,</span> <span class="s">&#34;\\1&#34;</span><span class="p">))</span>
</span></span></code></pre></td></tr></table>
</div>
</div><div class="highlight"><div class="chroma">
<table class="lntable"><tr><td class="lntd">
<pre tabindex="0" class="chroma"><code><span class="lnt">1
</span><span class="lnt">2
</span></code></pre></td>
<td class="lntd">
<pre tabindex="0" class="chroma"><code class="language-r" data-lang="r"><span class="line"><span class="cl"><span class="n">csv</span> <span class="n">doc</span> <span class="n">gif</span> <span class="n">jpg</span> <span class="n">png</span> <span class="n">xls</span>
</span></span><span class="line"><span class="cl"> <span class="m">67</span>  <span class="m">51</span>  <span class="m">21</span>   <span class="m">4</span>   <span class="m">1</span>   <span class="m">2</span>
</span></span></code></pre></td></tr></table>
</div>
</div><p>140多篇动态报道，只有67篇用了网页表格附件，有53篇是MS office文档，还有26篇直接贴图了事。</p>
<p>真是一叶落而知天下秋。卫生系统的数据化水准，跟金融系统比起来真是天上地下，一个站着，一个躺着。</p>
<p><a href="/post/infectdis_stat2/">待续</a></p>
<hr>
<!-- {% raw %} -->
<figure class="center"><img src="https://gh-1251443721.cos.ap-chengdu.myqcloud.com/QRcode.jpg"
         alt="扫码关注" width="30%"/><figcaption>
            <h4>扫码关注我的公众号</h4>
        </figcaption>
</figure>
<!-- {% endraw %} -->
    </div>

    <div class="post-copyright">
  <p class="copyright-item">
    <span class="item-title">文章作者</span>
    <span class="item-content">迷幻主义搬砖号子</span>
  </p>
  <p class="copyright-item">
    <span class="item-title">上次更新</span>
    <span class="item-content">
        2021-05-09
        <a href="https://github.com/madlogos/hugo-blog.git/commit/f3e8c5ae54f54c3eea5e759b3bce5a3facc97c88" title="changes on 2021年 5月 9日 星期日 21时28分26秒 CST">(f3e8c5a)</a>
    </span>
  </p>
  <p class="copyright-item">
      <span class="item-title">原始文档</span>
      <span class="item-content"><a class="link-to-markdown" href="https://madlogos.github.io/post/infectdis_stat1/index.md" target="_blank">查看本文 Markdown 版本 »</a></span>
    </p>
  <p class="copyright-item">
    <span class="item-title">许可协议</span>
    <span class="item-content"><a rel="license noopener" href="https://creativecommons.org/licenses/by-nc-nd/4.0/" target="_blank">CC BY-NC-ND 4.0</a></span>
  </p>
</div>
<div class="post-reward">
  <input type="checkbox" name="reward" id="reward" hidden />
  <label class="reward-button" for="reward">赞赏支持</label>
  <div class="qr-code">
    
    <label class="qr-code-image" for="reward">
        <img class="image" src="/img/reward/wechat.png">
        <span>微信打赏</span>
      </label>
    <label class="qr-code-image" for="reward">
        <img class="image" src="/img/reward/alipay.png">
        <span>支付宝打赏</span>
      </label>
  </div>
</div><footer class="post-footer">
      <div class="post-tags">
          <a href="/tags/%E4%BC%A0%E6%9F%93%E7%97%85/">传染病</a>
          <a href="/tags/%E7%96%AB%E6%83%85%E6%8A%A5%E5%91%8A/">疫情报告</a>
          <a href="/tags/rvest/">rvest</a>
          </div>
      <nav class="post-nav">
        <a class="prev" href="/post/infectdis_stat2/">
            <i class="iconfont icon-left"></i>
            <span class="prev-text nav-default">一次不太成功的搬砖（中）：rvest爬取法定传染病疫情月报数据</span>
            <span class="prev-text nav-mobile">上一篇</span>
          </a>
        <a class="next" href="/post/defensive-assassin/">
            <span class="next-text nav-default">礼法两难的辱母杀人案</span>
            <span class="next-text nav-mobile">下一篇</span>
            <i class="iconfont icon-right"></i>
          </a>
      </nav>
    </footer>
  </article>
        </div>
        <div id="disqus_thread"></div>
    <script type="text/javascript">
    (function() {
      
      
      if (window.location.hostname === 'localhost') return;

      var dsq = document.createElement('script'); dsq.type = 'text/javascript'; dsq.async = true;
      var disqus_shortname = 'madlogos-gh';
      dsq.src = '//' + disqus_shortname + '.disqus.com/embed.js';
      (document.getElementsByTagName('head')[0] || document.getElementsByTagName('body')[0]).appendChild(dsq);
    })();
    </script>
    <noscript>Please enable JavaScript to view the <a href="http://disqus.com/?ref_noscript">comments powered by Disqus.</a></noscript>

  

  

      </div>
    </main>

    <footer id="footer" class="footer">
      <div class="social-links">
      <a href="mailto:madlogos@gmail.com" class="iconfont icon-email" title="email"></a>
      <a href="https://stackoverflow.com/users/5455754/madlogos?tab=profile" class="iconfont icon-stack-overflow" title="stack-overflow"></a>
      <a href="https://twitter.com/madlogos" class="iconfont icon-twitter" title="twitter"></a>
      <a href="https://www.facebook.com/madlogos" class="iconfont icon-facebook" title="facebook"></a>
      <a href="http://www.linkedin.com/in/yiying-wang/" class="iconfont icon-linkedin" title="linkedin"></a>
      <a href="https://plus.google.com/&#43;%E6%B1%AA%E8%BD%B6%E9%A2%96madlogos" class="iconfont icon-google" title="google"></a>
      <a href="http://github.com/madlogos" class="iconfont icon-github" title="github"></a>
      <a href="https://weibo.com/madlogos/" class="iconfont icon-weibo" title="weibo"></a>
      <a href="https://www.zhihu.com/people/madlogos/" class="iconfont icon-zhihu" title="zhihu"></a>
      <a href="https://www.douban.com/people/Jandeaux/" class="iconfont icon-douban" title="douban"></a>
      <a href="https://jandeaux.tumblr.com" class="iconfont icon-tumblr" title="tumblr"></a>
      <a href="https://www.instagram.com/jandeaux/" class="iconfont icon-instagram" title="instagram"></a>
      <a href="https://gitlab.com/madlogos" class="iconfont icon-gitlab" title="gitlab"></a>
      <a href="https://space.bilibili.com/384080442" class="iconfont icon-bilibili" title="bilibili"></a>
  <a href="https://madlogos.github.io/index.xml" type="application/rss+xml" class="iconfont icon-rss" title="rss"></a>
</div>

<div class="copyright">
  <span class="power-by">
    由 <a class="hexo-link" href="https://gohugo.io">Hugo</a> 强力驱动
  </span>
  <span class="division">|</span>
  <span class="theme-info">
    主题 - 
    <a class="theme-link" href="https://github.com/olOwOlo/hugo-theme-even">Even</a>
  </span>

  <div class="busuanzi-footer">
    <span id="busuanzi_container_site_pv"> 本站总访问量 <span id="busuanzi_value_site_pv"><img src="/img/spinner.svg" alt="spinner.svg"/></span> 次 </span>
      <span class="division">|</span>
    <span id="busuanzi_container_site_uv"> 本站总访客数 <span id="busuanzi_value_site_uv"><img src="/img/spinner.svg" alt="spinner.svg"/></span> 人 </span>
  </div>

  <span class="copyright-year">
    &copy; 
    2017 - 
    2022
    
    <span class="author">madlogos</span>
  </span>
</div>

    </footer>

    <div class="back-to-top" id="back-to-top">
      <i class="iconfont icon-up"></i>
    </div>
  </div>
  
  <script src="https://cdn.jsdelivr.net/npm/jquery@3.6.0/dist/jquery.js" integrity="sha256-H+K7U5CnXl1h5ywQfKtSj8PCmoN9aaq30gDh27Xc0jk=" crossorigin="anonymous"></script>
  <script src="https://cdn.jsdelivr.net/npm/slideout@1.0.1/dist/slideout.min.js" integrity="sha256-t+zJ/g8/KXIJMjSVQdnibt4dlaDxc9zXr/9oNPeWqdg=" crossorigin="anonymous"></script>
  <script src="https://cdn.jsdelivr.net/npm/@fancyapps/fancybox@3.5.7/dist/jquery.fancybox.min.js" integrity="sha256-yt2kYMy0w8AbtF89WXb2P1rfjcP/HTHLT7097U8Y5b8=" crossorigin="anonymous"></script><script></script><script src="https://cdn.jsdelivr.net/npm/raphael@2.3.0/raphael.min.js" integrity="sha256-TabprKdeNXbSesCWLMrcbWSDzUhpAdcNPe5Q53rn9Yg=" crossorigin="anonymous"></script> <script src="https://cdn.jsdelivr.net/npm/flowchart.js@1.12.1/release/flowchart.min.js" integrity="sha256-ANSuVJkHZftRURALG24omixaZG+Sb51/+JY6EDa7MdE=" crossorigin="anonymous"></script><script></script><script src="https://cdn.jsdelivr.net/npm/webfontloader@1.6.28/webfontloader.js" integrity="sha256-4O4pS1SH31ZqrSO2A/2QJTVjTPqVe+jnYgOWUVr7EEc=" crossorigin="anonymous"></script> <script src="https://cdn.jsdelivr.net/npm/snapsvg@0.5.1/dist/snap.svg-min.js" integrity="sha256-oI+elz+sIm+jpn8F/qEspKoKveTc5uKeFHNNVexe6d8=" crossorigin="anonymous"></script> <script src="https://cdn.jsdelivr.net/npm/underscore@1.10.2/underscore-min.js" integrity="sha256-av1TvywtZ4ZqyCj/6HdtCHSJdn80HAzTgEBTJt/O8uc=" crossorigin="anonymous"></script> <script src="https://cdn.jsdelivr.net/npm/@rokt33r/js-sequence-diagrams@2.0.6-2/dist/sequence-diagram-min.js" integrity="sha256-eadHf9g1REH9Wvp2FLV/D9vKNvQUFKuVPgWFvmMQxBE=" crossorigin="anonymous"></script>
    <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/@rokt33r/js-sequence-diagrams@2.0.6-2/dist/sequence-diagram-min.css" integrity="sha256-On01v36B8LRDuL2tqhqs7Gb3Cm/NIpsLFy4OarOodUA=" crossorigin="anonymous">



<script type="text/javascript" src="/js/main.min.4ae89da218555efa0e7093a20b92017d2e1202b66fff9fc2edf4cb8d44b44c6e.js"></script>
  <script type="text/javascript">
    window.MathJax = {
      tex: {
        inlineMath: [['$','$'], ['\\(','\\)']],
        tags: 'ams',
        }
    };
  </script>
  <script async src="https://cdn.jsdelivr.net/npm/mathjax@3.0.5/es5/tex-mml-chtml.js" integrity="sha256-HGLuEfFcsUJGhvB8cQ8nr0gai9EucOOaIxFw7qxmd+w=" crossorigin="anonymous"></script>


<script type="application/javascript">
var doNotTrack = false;
if (!doNotTrack) {
	window.ga=window.ga||function(){(ga.q=ga.q||[]).push(arguments)};ga.l=+new Date;
	ga('create', 'UA-68964085-2', 'auto');
	ga('set', 'anonymizeIp', true);
	ga('send', 'pageview');
}
</script>
<script async src='https://www.google-analytics.com/analytics.js'></script>
<script id="baidu_analytics">
  var _hmt = _hmt || [];
  (function() {
    if (window.location.hostname === 'localhost') return;
    var hm = document.createElement("script"); hm.async = true;
    hm.src = "https://hm.baidu.com/hm.js?f221e23428218df65adeefe00f85e7ed";
    var s = document.getElementsByTagName("script")[0];
    s.parentNode.insertBefore(hm, s);
  })();
</script>

<script id="baidu_push">
  (function(){
    if (window.location.hostname === 'localhost') return;
    var bp = document.createElement('script'); bp.async = true;
    var curProtocol = window.location.protocol.split(':')[0];
    if (curProtocol === 'https') {
      bp.src = 'https://zz.bdstatic.com/linksubmit/push.js';
    }
    else {
      bp.src = 'http://push.zhanzhang.baidu.com/push.js';
    }
    var s = document.getElementsByTagName("script")[0];
    s.parentNode.insertBefore(bp, s);
  })();
</script>


<script src="/js/custom.js"></script>


</body>
</html>
